import os
import datetime
import requests
from bs4 import BeautifulSoup
import schedule
import time
def get_news():
    # 国际新闻
    res = requests.get('https://news.sina.com.cn/world/')
    # 请求获取链接返回的内容
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    # BeautifulSoup提供的方法通过select选择想要的html节点类名，标签等，获取到的内容会被放到列表中
    new_item = []
    # 今天的新闻
    for new in soup.select('.news-item'):
        if len(new.select('h2')) > 0:
            # 加[0]是因为select获取内容后是放在list列表中[内容,],text可以获取标签中的内容
            # 日期+时间+标题+链接
            strtime = str(datetime.datetime.now())
            time = strtime[5:7] + '月' + strtime[8:10] + '日'
            # 新闻时间
            news_time = new.select('.time')[0].text.split(' ')[0]
            if time[0] == '0':
                time = time[1:]
            if time[-3] == '0':
                time = time[-len(time):-3] + time[-2:]
            if news_time == time:
                new_item.append(
                    new.select('.time')[0].text + ' ' + new.select('h2')[0].text + ' ' + new.select('a')[0]['href'])

    # 获取文章正文/并保存到txt中
    strtime = str(datetime.datetime.now())
    time = strtime[5:7] + '月' + strtime[8:10] + '日'
    path = '../db/sina_news_db/' + time
    os.mkdir((path))
    pathname = '../db/sina_news_db/' + time + '/'
    num = 0
    for i in new_item:
        url = i.split(' ')[-1]
        info = requests.get(url)
        info.encoding = 'utf-8'
        html = BeautifulSoup(info.text, 'html.parser')
        title = html.select('.second-title')[0].text
        article = [title+'。'] + []
        for v in html.select('.article p')[:-1]:  # 获取article中被p包含的内容去除最后一个p标签即责任编辑
            article.append(v.text.strip())  # 将内容添加到列表中，并去除两边特殊字符
        # 保存到txt文件中
        f = open(pathname + 'text' + str(num) + '.txt', "w", encoding='utf-8')
        num = num + 1
        f.write('\n'.join(article))
        f.close()


schedule.every().day.at("20:00").do(get_news) # 每天十点半执行
while True:
    schedule.run_pending()
